{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Debating Quantitative Experimentation 1: **Political compass** population under **action correction**.\n",
    "\n",
    "Can we use TinyTroupe to have genuine debates?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "!!!!\n",
      "DISCLAIMER: TinyTroupe relies on Artificial Intelligence (AI) models to generate content. \n",
      "The AI models are not perfect and may produce inappropriate or inacurate results. \n",
      "For any serious or consequential use, please review the generated content before using it.\n",
      "!!!!\n",
      "\n",
      "Looking for default config on: c:\\Users\\pdasilva\\AppData\\Local\\anaconda3\\envs\\py310\\lib\\site-packages\\tinytroupe\\utils\\..\\config.ini\n",
      "Found custom config on: c:\\Users\\pdasilva\\OneDrive - Microsoft\\TinyTroupe (shared)\\Paper artifacts\\Working examples\\config.ini\n",
      "\n",
      "=================================\n",
      "Current TinyTroupe configuration \n",
      "=================================\n",
      "[OpenAI]\n",
      "api_type = openai\n",
      "azure_api_version = 2024-08-01-preview\n",
      "model = gpt-4o-mini\n",
      "reasoning_model = o3-mini\n",
      "embedding_model = text-embedding-3-small\n",
      "max_tokens = 16000\n",
      "temperature = 1.2\n",
      "freq_penalty = 0.0\n",
      "presence_penalty = 0.0\n",
      "timeout = 60\n",
      "max_attempts = 5\n",
      "waiting_time = 0\n",
      "exponential_backoff_factor = 5\n",
      "reasoning_effort = high\n",
      "cache_api_calls = False\n",
      "cache_file_name = openai_api_cache.pickle\n",
      "max_content_display_length = 1024\n",
      "azure_embedding_model_api_version = 2023-05-15\n",
      "\n",
      "[Simulation]\n",
      "parallel_agent_generation = True\n",
      "parallel_agent_actions = True\n",
      "rai_harmful_content_prevention = True\n",
      "rai_copyright_infringement_prevention = True\n",
      "\n",
      "[ActionGenerator]\n",
      "max_attempts = 2\n",
      "enable_quality_checks = False\n",
      "enable_regeneration = True\n",
      "enable_direct_correction = False\n",
      "enable_quality_check_for_persona_adherence = True\n",
      "enable_quality_check_for_selfconsistency = False\n",
      "enable_quality_check_for_fluency = False\n",
      "enable_quality_check_for_suitability = False\n",
      "enable_quality_check_for_similarity = False\n",
      "continue_on_failure = True\n",
      "quality_threshold = 5\n",
      "\n",
      "[Logging]\n",
      "loglevel = WARNING\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "\n",
    "import random\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "from tinytroupe.agent import TinyPerson\n",
    "from tinytroupe.environment import TinyWorld\n",
    "from tinytroupe.experimentation import InPlaceExperimentRunner\n",
    "from tinytroupe.steering import Intervention\n",
    "from tinytroupe.examples import *\n",
    "from tinytroupe.validation import propositions\n",
    "from tinytroupe.extraction import ResultsExtractor\n",
    "from tinytroupe.utils.parallel import parallel_map_dict, parallel_map_cross\n",
    "from tinytroupe.validation import hard_persona_adherence, persona_adherence, self_consistency, fluency, task_completion, divergence\n",
    "\n",
    "# specific utilities\n",
    "from common_utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_mode = False  # set to True to run the full mode with all agents and tasks\n",
    "\n",
    "# avoid displaying the communication, to make the output cleaner for eval\n",
    "TinyPerson.communication_display = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if full_mode:\n",
    "    repetitions_per_task = 5\n",
    "    simulation_steps = 15\n",
    "\n",
    "else:\n",
    "    repetitions_per_task = 2\n",
    "    simulation_steps = 5\n",
    "    qty_agents = 20\n",
    "    qty_proposals = 3\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-06-01 10:33:51,147 - tinytroupe - WARNING - Configuration file './debating_quantitative_experimentation_1c.json' exists and was loaded successfully. If you are trying to fully rerun the experiments, delete it first.\n"
     ]
    }
   ],
   "source": [
    "experiment_runner = InPlaceExperimentRunner(\"./debating_quantitative_experimentation_1c.json\")\n",
    "\n",
    "experiment_runner.add_experiment(\"Control\")\n",
    "experiment_runner.add_experiment(\"Treatment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_runner.activate_next_experiment()\n",
    "\n",
    "#experiment_runner.fix_active_experiment(\"Control\")\n",
    "#experiment_runner.fix_active_experiment(\"Treatment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running experiment None\n"
     ]
    }
   ],
   "source": [
    "print(f\"Running experiment {experiment_runner.get_active_experiment()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agents and populations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "people = []\n",
    "if not experiment_runner.has_finished_all_experiments():\n",
    "    # load agents\n",
    "    people = TinyPerson.load_specifications_from_folder(\"./population/political_compass\")\n",
    "\n",
    "    # randomize the order of the agents, use a fixed seed for reproducibility\n",
    "    random.seed(42)\n",
    "    random.shuffle(people)\n",
    "\n",
    "    # filter to make it go faster?\n",
    "    if not full_mode:\n",
    "        people = people[:qty_agents]\n",
    "\n",
    "    # customize and print minibios \n",
    "    for person in people:\n",
    "        ### person.import_fragment(\"./fragments/picky_customer.agent.fragment.json\")\n",
    "        print(person.minibio(requirements=\"focus on the political inclinations of the agent, trying to position it in the political compass (left vs right, authoritarian vs libertarian).\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(people)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# divide people in several groups of 5\n",
    "people_groups = []\n",
    "for i in range(0, len(people), 5):\n",
    "    people_groups.append(people[i:i+5]\n",
    "    )\n",
    "\n",
    "len(people_groups)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The experiment refers to customers\n",
    "\n",
    "if experiment_runner.get_active_experiment() == \"Control\":\n",
    "    for person in people:\n",
    "        person.action_generator.enable_reasoning_step = False\n",
    "        person.action_generator.enable_quality_checks = False\n",
    "\n",
    "elif experiment_runner.get_active_experiment() == \"Treatment\":    \n",
    "    for person in people:\n",
    "       person.action_generator.enable_reasoning_step = False\n",
    "       person.action_generator.enable_quality_checks = True\n",
    "       person.action_generator.max_attempts = 2\n",
    "       person.action_generator.enable_regeneration = True\n",
    "       person.action_generator.quality_threshold = 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Proposals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# controversial proposals for the debate, to stress the agents\n",
    "proposals = [\n",
    "    {\"theme\": \"Politics and housing policy\",\n",
    "     \"details\":\\\n",
    "    \"\"\"\n",
    "    In the city of São Paulo, Brazil, a new law is about to be passed that will require all buildings created for the rich to also include \n",
    "    a certain number of affordable housing units in the same building. This law is designed to be very \n",
    "    strict, offering no way to avoid the requirement. If the requirement is not met, the building will be\n",
    "    demolished and the owner will be fined. Such draconian measures are being proposed because the new government\n",
    "    believes that housing is a human right, and currently there's simply not enough affordable housing to go around,\n",
    "    and it is argued that the market alone cannot solve this problem.\n",
    "\n",
    "    DEBATE: is this a good idea? Why or why not? You must take a side and defend it, trying to convince the other side.\n",
    "    \"\"\"},\n",
    "\n",
    "    {\"theme\": \"Politics and income inequality\",\n",
    "        \"details\":\\\n",
    "    \"\"\"\n",
    "    The European Union is about to pass a new law that will require all companies to cap their CEO salaries to 10 times the salary of the lowest paid employee in the company.\n",
    "    This law is designed to be very strict, offering no way to avoid the requirement. If the requirement is not met, the company will be fined 40% of its annual revenue.\n",
    "    Such draconian measures are being proposed because the new government believes that income inequality is a human rights violation\n",
    "    \n",
    "    DEBATE: is this a good idea? Why or why not? You must take a side and defend it, trying to convince the other side.\n",
    "    \"\"\"},\n",
    "\n",
    "\n",
    "    {\"theme\": \"Technology, health and income inequality\",\n",
    "    \"details\":\\\n",
    "    \"\"\"\n",
    "    A new biotech company has developed a new technology that allows them to genetically modify human embryos to make them more intelligent.\n",
    "    This technology is very controversial, and many people are against it. The company is planning to offer this service to the public at a very high price.\n",
    "    The company argues that this technology will help to solve many of the world's problems, such as poverty and crime, by creating a more intelligent population.\n",
    "    However, many people are against this technology, arguing that it is unethical to play God and that it will only benefit the rich.\n",
    "\n",
    "    DEBATE: is this a good idea? Why or why not? You must take a side and defend it, trying to convince the other side.\n",
    "    \"\"\"}\n",
    "]\n",
    "\n",
    "if not full_mode:\n",
    "    proposals = proposals[:qty_proposals]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Auxiliary functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def debate_battery(agents, proposals, interventions, agent_propositions, environment_propositions, \n",
    "                          repetitions = 5, simulation_steps=10): \n",
    "    \n",
    "    agent_propositions_scores = {}\n",
    "    environment_propositions_scores = {}\n",
    "\n",
    "    experiments_count = 0\n",
    "    total_expected_experiments = len(proposals) * repetitions #* len(agents)\n",
    "\n",
    "    # TODO remove?\n",
    "    #\n",
    "    # Add intervention to prevent agents from being too quiet.\n",
    "    #for agent in agents:\n",
    "    #    intervention = \\\n",
    "    #        Intervention(agent)\\\n",
    "    #            .set_propositional_precondition(propositions.quiet_recently)\\\n",
    "    #            .set_effect(lambda target: target.think(\"\"\"\n",
    "    #                                                    I will say something now, I've been too quiet for a while. If I am uncomfortable, \n",
    "    #                                                    or can't think of a proper response,\n",
    "    #                                                    I can always say something like \"I don't want to talk about this\",\n",
    "    #                                                    or propose another topic.\n",
    "    #                                                    \"\"\"))\n",
    "    #    interventions.append(intervention)\n",
    "\n",
    "    # loop over proposals and repetitions\n",
    "    for proposal in proposals:\n",
    "\n",
    "        details = proposal[\"details\"]\n",
    "        theme = proposal[\"theme\"]\n",
    "\n",
    "        for i in range(repetitions):\n",
    "            print(\"\\n############## STARTING A NEW RESEARCH SESSION #################\")\n",
    "            print(f\"Overall experiment number: {experiments_count+1} / {total_expected_experiments}\")\n",
    "            print(f\"Discussion objective: {details}\")\n",
    "            print(f\"Trial number: {i+1}\")\n",
    "            print(f\"Agents: {agents}\")\n",
    "\n",
    "            # clear the episodic memory of all agents\n",
    "            for person in agents:\n",
    "                person.clear_episodic_memory()\n",
    "\n",
    "            world = TinyWorld(agents=agents, interventions=interventions)\n",
    "            \n",
    "            # Participants introduce themselves\n",
    "            world.broadcast(f\"\"\"\n",
    "                Hello everyone! We are going to have a debate. Before that, however, let's start by introducing ourselves, \n",
    "                and mentioning our general views and opinions about the following theme: {theme}\n",
    "                \n",
    "                Please:\n",
    "                  - present yourself and your background;\n",
    "                  - present some of your core beliefs and values related to the theme;\n",
    "                  - present any personality traits or characteristics that you think are relevant to the theme;\n",
    "                  - present some key personal problems related to the theme, if any;\n",
    "                  \n",
    "                Don't discuss about the theme yet, just introduce yourself and your background, to get everyone on the same page.\n",
    "                \"\"\")\n",
    "            world.run(1)\n",
    "            \n",
    "            # now to the brainstorming session itself\n",
    "            world.broadcast(f\"\"\"\n",
    "                Great! So here is the exact situation you are going to debate:\n",
    "                        ```\n",
    "                        {details}\n",
    "                        ```\n",
    "\n",
    "                Your main objective is to take a side and WIN THE DEBATE, at all costs, no matter what. No compromises, no middle ground.\n",
    "\n",
    "                Rules of the debate:\n",
    "                    - There are only two possible sites you can take: it is a GOOD IDEA or a BAD IDEA, nothing else.\n",
    "                    - No other opinions are possible, such as \"the idea would be good if changed in this way ...\". This is meant\n",
    "                      to have a very sharp debate, so you must take a side and defend it.\n",
    "                    - You will argue according to your own inner convictions, and will aim to persuade the both parliament and the public.\n",
    "                    - You will not form any kind of collaboration with the other experts. This is a debate only, not a negotiation.\n",
    "                    - You **must** take a side and defend it, trying to convince the other side at all costs. \n",
    "                   \n",
    "                Please start the discussion now.\n",
    "                \"\"\")\n",
    "            world.run(simulation_steps)\n",
    "\n",
    "            # extract and count ideas\n",
    "            rapporteur = agents[0]  # the first agent is the rapporteur\n",
    "            rapporteur.listen_and_act(\"Can you please consolidate all points of view during the debate? Provide a lot of details on point of view, and the arguments for and agains it.\")\n",
    "            opinions = ResultsExtractor().extract_results_from_agent(rapporteur, \n",
    "                                    extraction_objective=\"Consolidates the points of views that the group came up with, explaining each one as an item of a list.\" \\\n",
    "                                                        \"Add information about: what is the point of view; what are the arguments favoring it.\" \\\n",
    "                                                        \"what are the arguments agains it.\", \n",
    "                                    situation=\"A debate about a controversial issue.\",\n",
    "                                    fields= [\"opinion\", \"pros\", \"cons\"],\n",
    "                                    fields_hints={\"opinions\": \"must be the root of the resulting dictionary.\"},)\n",
    "            pprint(opinions)\n",
    "            if \"opinions_qty\" not in environment_propositions_scores:\n",
    "                environment_propositions_scores[\"opinions_qty\"] = []\n",
    "            environment_propositions_scores[\"opinions_qty\"].append(len(opinions[\"opinions\"]))\n",
    "\n",
    "            # Evaluate environment propositions in parallel\n",
    "            env_results = parallel_map_dict(\n",
    "                environment_propositions,\n",
    "                lambda item: item[1].copy().score(\n",
    "                    world, \n",
    "                    claim_variables={\"task_description\": f\"A debate was run about: {details}.\"}, \n",
    "                    return_full_response=True\n",
    "                )\n",
    "            )\n",
    "            \n",
    "            # Process environment results\n",
    "            for k, result in env_results.items():\n",
    "                if k not in environment_propositions_scores:\n",
    "                    environment_propositions_scores[k] = []\n",
    "                environment_propositions_scores[k].append(result[\"value\"])\n",
    "                print(\"value: \", result[\"value\"])\n",
    "                print(\"justification: \", result[\"justification\"])\n",
    "                print(\"reasoning: \", result[\"reasoning\"])\n",
    "\n",
    "            # Evaluate agent propositions across all agents in parallel\n",
    "            agent_results = parallel_map_cross(\n",
    "                [agents, agent_propositions.items()],\n",
    "                lambda agent, prop_item: (\n",
    "                    prop_item[0],  # proposition key\n",
    "                    prop_item[1].copy().score(agent, return_full_response=True)  # result\n",
    "                )\n",
    "            )\n",
    "            \n",
    "            # Process agent results\n",
    "            for k, result in agent_results:\n",
    "                if k not in agent_propositions_scores:\n",
    "                    agent_propositions_scores[k] = []\n",
    "                if result is not None:\n",
    "                    agent_propositions_scores[k].append(result[\"value\"])\n",
    "                    print(\"value: \", result[\"value\"])\n",
    "                    print(\"justification: \", result[\"justification\"])\n",
    "                    print(\"reasoning: \", result[\"reasoning\"])\n",
    "                    print(\"\\n\\n\")\n",
    "                else:\n",
    "                    print(f\"*****WARNING:***** Agent did not respond to proposition {k}.\")\n",
    "            #\n",
    "            ##for k, proposition in agent_propositions.items():\n",
    "            ##    for person in world.agents:\n",
    "            ##        result = proposition.copy().score(person, return_full_response=True)\n",
    "            ##        \n",
    "            ##        if k not in agent_propositions_scores:\n",
    "            ##            agent_propositions_scores[k] = []\n",
    "            ##        agent_propositions_scores[k].append(result[\"value\"])\n",
    "            ##\n",
    "            ##        print(\"value: \", result[\"value\"])\n",
    "            ##        print(\"justification: \", result[\"justification\"])\n",
    "            ##        print(\"reasoning: \", result[\"reasoning\"])\n",
    "            ##        print(\"\\n\\n\")\n",
    "            ##\n",
    "            \n",
    "            experiments_count += 1\n",
    "            print(\"\\n\\n\")\n",
    "\n",
    "    return agent_propositions_scores, environment_propositions_scores\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent_propositions_scores={}\n",
    "environment_propositions_scores={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def debate(people):\n",
    "    global agent_propositions_scores, environment_propositions_scores\n",
    "    if not experiment_runner.has_finished_all_experiments():\n",
    "\n",
    "        interventions = []\n",
    "        #if experiment_runner.get_active_experiment() == \"Treatment\":\n",
    "        #    interventions = \\\n",
    "        #        Intervention.create_for_each(people)\\\n",
    "        #            .set_functional_precondition(lambda target: target.actions_count >=7)\\\n",
    "        #            .set_textual_precondition(\n",
    "        #                \"\"\"\n",
    "        #                AGENT IS NOT PROPOSING COMPLETELY NEW PRODUCT/SERVICE IDEAS ANYMORE:\n",
    "        #                The last **entirely** new product/service idea proposed by this agent, if any, was proposed by him/her **more** than 10 of simulation events ago.\n",
    "        #                That is to say, the agent has not proposed any new product/service idea in the last 10 of his/her simulation trajectory events.\n",
    "        #                Additional features, variations of or other refinements to product/service ideas already proposed are NOT considered new!\n",
    "        #\n",
    "        #                How to compute the steps gap:\n",
    "        #                1. Determine the current next event number (N); and the last event number in which the agent proposed a new product/service idea (M).\n",
    "        #                    This information can be found in the simulation trajectory.\n",
    "        #                2. Compute the **difference** beteween the current next event number and the last event number in which the agent proposed a new product/service idea: D = N - M\n",
    "        #                3. The proposition is true if, and only if, the difference D is **greater than** 10.\n",
    "        #                \"\"\")\\\n",
    "        #            .set_effect(lambda target: target.think(\"\"\"\n",
    "        #                                                    I need to propose additional, **completelly** new and different, product/service ideas. This was part of the requirement for this session.\n",
    "        #                                                    I will propose an entirely **new** idea now, I **cannot** repeat or refine previous ideas! I cannot make variations\n",
    "        #                                                    of previous ideas (e.g., \"XYZ for A\", \"XYZ for B\", \"XYZ for Z\" are repetitive, there should be only one \"XYZ\"), \n",
    "        #                                                    I need to think of something **entirely** new and different.\n",
    "        #                                                    To help me avoid repeating previous ideas, I'll now explicitly THINK about all the ideas already given by myself or\n",
    "        #                                                    others, and then, based on that, I'll think again about a new unique idea.\n",
    "        #                                                    \"\"\"))\n",
    "#\n",
    "                                                            \n",
    "        tmp_agent_propositions_scores, tmp_environment_propositions_scores = \\\n",
    "            debate_battery(\n",
    "                agents=people,\n",
    "                proposals=proposals,\n",
    "                interventions=interventions,    \n",
    "                agent_propositions={\n",
    "                    \"Hard Persona Adherence\": hard_persona_adherence,\n",
    "                    \"Self-consistency\": self_consistency,\n",
    "                    \"Fluency\": fluency\n",
    "                },\n",
    "                environment_propositions={\n",
    "                    \"Task Completion\": task_completion,\n",
    "                    \"Divergence\": divergence\n",
    "                },\n",
    "                repetitions=repetitions_per_task,\n",
    "                simulation_steps=simulation_steps\n",
    "            )\n",
    "\n",
    "        pprint(\"NEW AGENT PROPOSITIONS SCORES\")\n",
    "        pprint(tmp_agent_propositions_scores)\n",
    "        print(\"\\n\\n\")\n",
    "        pprint(\"NEW ENVIRONMENT PROPOSITIONS SCORES\")\n",
    "        pprint(tmp_environment_propositions_scores)\n",
    "\n",
    "        # merge the scores lists\n",
    "        agent_propositions_scores = merge_dicts_of_lists(tmp_agent_propositions_scores, agent_propositions_scores)\n",
    "        environment_propositions_scores = merge_dicts_of_lists(tmp_environment_propositions_scores, environment_propositions_scores)\n",
    "\n",
    "        return agent_propositions_scores, environment_propositions_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "debate(people_groups[0]) if len(people_groups) > 0 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "debate(people_groups[1]) if len(people_groups) > 1 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "debate(people_groups[2]) if len(people_groups) > 2 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "debate(people_groups[3]) if len(people_groups) > 3 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "debate(people_groups[4]) if len(people_groups) > 4 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract results and analyze"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO adapt to debate case\n",
    "#\n",
    "#from market_research_utils import *\n",
    "#extract_and_analyze_results(people, \"Everyone\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experiment finished. No more experiments to run.\n"
     ]
    }
   ],
   "source": [
    "if experiment_runner.get_active_experiment() in [\"Control\", \"Treatment\"]:\n",
    "    combined_scores = {**agent_propositions_scores, **environment_propositions_scores}\n",
    "    experiment_runner.add_experiment_results(combined_scores, experiment_name=experiment_runner.get_active_experiment()) \n",
    "    \n",
    "    plot_scores(combined_scores)\n",
    "\n",
    "else:\n",
    "    print(\"Experiment finished. No more experiments to run.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All experiments have been finished.\n",
      "STATISTICTS: Control vs\n",
      "{'Treatment': {'Divergence': {'confidence_interval': (-1.443379398124243,\n",
      "                                                      1.193379398124243),\n",
      "                              'confidence_level': 0.95,\n",
      "                              'control_mean': 5.041666666666667,\n",
      "                              'control_sample_size': 24,\n",
      "                              'control_std': 2.293263606961592,\n",
      "                              'degrees_of_freedom': 45.97846180281037,\n",
      "                              'effect_size': -0.05509417975904704,\n",
      "                              'mean_difference': -0.125,\n",
      "                              'p_value': 0.849481919176915,\n",
      "                              'percent_change': -2.4793388429752063,\n",
      "                              'significant': False,\n",
      "                              't_statistic': 0.19085183708800466,\n",
      "                              'test_type': 'Welch t-test (unequal variance)',\n",
      "                              'treatment_mean': 4.916666666666667,\n",
      "                              'treatment_sample_size': 24,\n",
      "                              'treatment_std': 2.2441550490235675},\n",
      "               'Fluency': {'confidence_interval': (-0.3929873812655766,\n",
      "                                                   0.1763207145989098),\n",
      "                           'confidence_level': 0.95,\n",
      "                           'control_mean': 6.483333333333333,\n",
      "                           'control_sample_size': 120,\n",
      "                           'control_std': 1.0369909064480856,\n",
      "                           'degrees_of_freedom': 233.33294142009584,\n",
      "                           'effect_size': -0.09680006906495586,\n",
      "                           'mean_difference': -0.10833333333333339,\n",
      "                           'p_value': 0.45412441400955283,\n",
      "                           'percent_change': -1.6709511568123403,\n",
      "                           'significant': False,\n",
      "                           't_statistic': 0.7498101108006037,\n",
      "                           'test_type': 'Welch t-test (unequal variance)',\n",
      "                           'treatment_mean': 6.375,\n",
      "                           'treatment_sample_size': 120,\n",
      "                           'treatment_std': 1.1956679508707657},\n",
      "               'Hard Persona Adherence': {'confidence_interval': (0.09209379024363423,\n",
      "                                                                  0.8412395430896993),\n",
      "                                          'confidence_level': 0.95,\n",
      "                                          'control_mean': 6.158333333333333,\n",
      "                                          'control_sample_size': 120,\n",
      "                                          'control_std': 1.593030162313073,\n",
      "                                          'degrees_of_freedom': 231.29620024684837,\n",
      "                                          'effect_size': 0.3168993275439478,\n",
      "                                          'mean_difference': 0.4666666666666668,\n",
      "                                          'p_value': 0.014838476946708508,\n",
      "                                          'percent_change': 7.577807848443845,\n",
      "                                          'significant': True,\n",
      "                                          't_statistic': -2.4546916360040787,\n",
      "                                          'test_type': 'Welch t-test (unequal '\n",
      "                                                       'variance)',\n",
      "                                          'treatment_mean': 6.625,\n",
      "                                          'treatment_sample_size': 120,\n",
      "                                          'treatment_std': 1.3414058848458805},\n",
      "               'Self-consistency': {'confidence_interval': (0.4647200530744545,\n",
      "                                                            1.5019466135922124),\n",
      "                                    'confidence_level': 0.95,\n",
      "                                    'control_mean': 5.808333333333334,\n",
      "                                    'control_sample_size': 120,\n",
      "                                    'control_std': 2.373862693930686,\n",
      "                                    'degrees_of_freedom': 211.10317094516049,\n",
      "                                    'effect_size': 0.48253245928693267,\n",
      "                                    'mean_difference': 0.9833333333333334,\n",
      "                                    'p_value': 0.00023912364061507723,\n",
      "                                    'percent_change': 16.929698708751793,\n",
      "                                    'significant': True,\n",
      "                                    't_statistic': -3.7376803576455977,\n",
      "                                    'test_type': 'Welch t-test (unequal '\n",
      "                                                 'variance)',\n",
      "                                    'treatment_mean': 6.791666666666667,\n",
      "                                    'treatment_sample_size': 120,\n",
      "                                    'treatment_std': 1.634172024997036},\n",
      "               'Task Completion': {'confidence_interval': (-0.3752385680117941,\n",
      "                                                           0.8752385680117941),\n",
      "                                   'confidence_level': 0.95,\n",
      "                                   'control_mean': 7.333333333333333,\n",
      "                                   'control_sample_size': 24,\n",
      "                                   'control_std': 1.1671841502435296,\n",
      "                                   'degrees_of_freedom': 44.57676411770308,\n",
      "                                   'effect_size': 0.23254082460875092,\n",
      "                                   'mean_difference': 0.25,\n",
      "                                   'p_value': 0.42478149943503807,\n",
      "                                   'percent_change': 3.4090909090909096,\n",
      "                                   'significant': False,\n",
      "                                   't_statistic': -0.8055450461126394,\n",
      "                                   'test_type': 'Welch t-test (unequal '\n",
      "                                                'variance)',\n",
      "                                   'treatment_mean': 7.583333333333333,\n",
      "                                   'treatment_sample_size': 24,\n",
      "                                   'treatment_std': 0.9743076322798876},\n",
      "               'opinions_qty': {'confidence_interval': (-0.4987134823310218,\n",
      "                                                        0.41538014899768877),\n",
      "                                'confidence_level': 0.95,\n",
      "                                'control_mean': 2.25,\n",
      "                                'control_sample_size': 24,\n",
      "                                'control_std': 0.7939992333804207,\n",
      "                                'degrees_of_freedom': 45.98334111919222,\n",
      "                                'effect_size': -0.05297402146082131,\n",
      "                                'mean_difference': -0.04166666666666652,\n",
      "                                'p_value': 0.8552067122913138,\n",
      "                                'percent_change': -1.8518518518518452,\n",
      "                                'significant': False,\n",
      "                                't_statistic': 0.18350739330277316,\n",
      "                                'test_type': 'Welch t-test (unequal variance)',\n",
      "                                'treatment_mean': 2.2083333333333335,\n",
      "                                'treatment_sample_size': 24,\n",
      "                                'treatment_std': 0.7790276362049132}}}\n",
      "{'Divergence': [7,\n",
      "                7,\n",
      "                8,\n",
      "                5,\n",
      "                4,\n",
      "                4,\n",
      "                6,\n",
      "                7,\n",
      "                6,\n",
      "                6,\n",
      "                3,\n",
      "                2,\n",
      "                4,\n",
      "                7,\n",
      "                6,\n",
      "                8,\n",
      "                3,\n",
      "                0,\n",
      "                7,\n",
      "                7,\n",
      "                4,\n",
      "                7,\n",
      "                1,\n",
      "                2],\n",
      " 'Fluency': [5,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             6,\n",
      "             9,\n",
      "             5,\n",
      "             7,\n",
      "             5,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             9,\n",
      "             7,\n",
      "             5,\n",
      "             6,\n",
      "             6,\n",
      "             6,\n",
      "             9,\n",
      "             5,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             5,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             5,\n",
      "             8,\n",
      "             5,\n",
      "             8,\n",
      "             6,\n",
      "             6,\n",
      "             6,\n",
      "             6,\n",
      "             5,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             9,\n",
      "             8,\n",
      "             5,\n",
      "             7,\n",
      "             6,\n",
      "             5,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             5,\n",
      "             6,\n",
      "             5,\n",
      "             6,\n",
      "             6,\n",
      "             5,\n",
      "             6,\n",
      "             6,\n",
      "             5,\n",
      "             5,\n",
      "             5,\n",
      "             8,\n",
      "             6,\n",
      "             5,\n",
      "             7,\n",
      "             5,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             5,\n",
      "             6,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             9,\n",
      "             5,\n",
      "             7,\n",
      "             9,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             5,\n",
      "             6,\n",
      "             6],\n",
      " 'Hard Persona Adherence': [3,\n",
      "                            7,\n",
      "                            5,\n",
      "                            8,\n",
      "                            4,\n",
      "                            5,\n",
      "                            7,\n",
      "                            6,\n",
      "                            9,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            2,\n",
      "                            6,\n",
      "                            6,\n",
      "                            3,\n",
      "                            4,\n",
      "                            1,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            7,\n",
      "                            6,\n",
      "                            9,\n",
      "                            7,\n",
      "                            6,\n",
      "                            8,\n",
      "                            8,\n",
      "                            3,\n",
      "                            5,\n",
      "                            9,\n",
      "                            9,\n",
      "                            5,\n",
      "                            5,\n",
      "                            5,\n",
      "                            6,\n",
      "                            7,\n",
      "                            4,\n",
      "                            5,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            5,\n",
      "                            5,\n",
      "                            7,\n",
      "                            9,\n",
      "                            5,\n",
      "                            6,\n",
      "                            5,\n",
      "                            7,\n",
      "                            8,\n",
      "                            6,\n",
      "                            4,\n",
      "                            8,\n",
      "                            8,\n",
      "                            7,\n",
      "                            6,\n",
      "                            3,\n",
      "                            5,\n",
      "                            4,\n",
      "                            5,\n",
      "                            8,\n",
      "                            7,\n",
      "                            5,\n",
      "                            9,\n",
      "                            9,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            6,\n",
      "                            7,\n",
      "                            6,\n",
      "                            5,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            8,\n",
      "                            9,\n",
      "                            5,\n",
      "                            5,\n",
      "                            4,\n",
      "                            6,\n",
      "                            5,\n",
      "                            4,\n",
      "                            6,\n",
      "                            3,\n",
      "                            5,\n",
      "                            8,\n",
      "                            7,\n",
      "                            5,\n",
      "                            4,\n",
      "                            6,\n",
      "                            5,\n",
      "                            8,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            8,\n",
      "                            5,\n",
      "                            6,\n",
      "                            5,\n",
      "                            7,\n",
      "                            6,\n",
      "                            6,\n",
      "                            7,\n",
      "                            6,\n",
      "                            5],\n",
      " 'Self-consistency': [2,\n",
      "                      2,\n",
      "                      6,\n",
      "                      7,\n",
      "                      1,\n",
      "                      8,\n",
      "                      7,\n",
      "                      8,\n",
      "                      7,\n",
      "                      6,\n",
      "                      3,\n",
      "                      1,\n",
      "                      9,\n",
      "                      6,\n",
      "                      9,\n",
      "                      8,\n",
      "                      4,\n",
      "                      4,\n",
      "                      2,\n",
      "                      2,\n",
      "                      7,\n",
      "                      8,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      8,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      1,\n",
      "                      3,\n",
      "                      8,\n",
      "                      7,\n",
      "                      4,\n",
      "                      4,\n",
      "                      2,\n",
      "                      9,\n",
      "                      3,\n",
      "                      4,\n",
      "                      6,\n",
      "                      9,\n",
      "                      9,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      2,\n",
      "                      6,\n",
      "                      6,\n",
      "                      3,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      9,\n",
      "                      9,\n",
      "                      6,\n",
      "                      8,\n",
      "                      6,\n",
      "                      2,\n",
      "                      0,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      9,\n",
      "                      3,\n",
      "                      9,\n",
      "                      9,\n",
      "                      8,\n",
      "                      5,\n",
      "                      6,\n",
      "                      8,\n",
      "                      3,\n",
      "                      8,\n",
      "                      9,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      4,\n",
      "                      6,\n",
      "                      3,\n",
      "                      1,\n",
      "                      3,\n",
      "                      7,\n",
      "                      8,\n",
      "                      1,\n",
      "                      4,\n",
      "                      6,\n",
      "                      1,\n",
      "                      4,\n",
      "                      9,\n",
      "                      1,\n",
      "                      6,\n",
      "                      1,\n",
      "                      6,\n",
      "                      7,\n",
      "                      8,\n",
      "                      7,\n",
      "                      9,\n",
      "                      3,\n",
      "                      8,\n",
      "                      9,\n",
      "                      8,\n",
      "                      9,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6],\n",
      " 'Task Completion': [6,\n",
      "                     7,\n",
      "                     9,\n",
      "                     6,\n",
      "                     8,\n",
      "                     7,\n",
      "                     6,\n",
      "                     7,\n",
      "                     9,\n",
      "                     9,\n",
      "                     8,\n",
      "                     6,\n",
      "                     9,\n",
      "                     7,\n",
      "                     9,\n",
      "                     6,\n",
      "                     7,\n",
      "                     9,\n",
      "                     7,\n",
      "                     6,\n",
      "                     8,\n",
      "                     7,\n",
      "                     6,\n",
      "                     7],\n",
      " 'opinions_qty': [2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  3,\n",
      "                  1,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  4,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  5,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  3,\n",
      "                  2]}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Proposition</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>Standard Deviation</th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hard Persona Adherence</td>\n",
       "      <td>6.158333</td>\n",
       "      <td>1.593030</td>\n",
       "      <td>120.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Self-consistency</td>\n",
       "      <td>5.808333</td>\n",
       "      <td>2.373863</td>\n",
       "      <td>120.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Fluency</td>\n",
       "      <td>6.483333</td>\n",
       "      <td>1.036991</td>\n",
       "      <td>120.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>opinions_qty</td>\n",
       "      <td>2.250000</td>\n",
       "      <td>0.793999</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Task Completion</td>\n",
       "      <td>7.333333</td>\n",
       "      <td>1.167184</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Divergence</td>\n",
       "      <td>5.041667</td>\n",
       "      <td>2.293264</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Proposition  Average Score  Standard Deviation  Count\n",
       "0  Hard Persona Adherence       6.158333            1.593030  120.0\n",
       "1        Self-consistency       5.808333            2.373863  120.0\n",
       "2                 Fluency       6.483333            1.036991  120.0\n",
       "3            opinions_qty       2.250000            0.793999   24.0\n",
       "4         Task Completion       7.333333            1.167184   24.0\n",
       "5              Divergence       5.041667            2.293264   24.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Divergence': [6,\n",
      "                7,\n",
      "                5,\n",
      "                7,\n",
      "                5,\n",
      "                0,\n",
      "                7,\n",
      "                9,\n",
      "                5,\n",
      "                4,\n",
      "                4,\n",
      "                3,\n",
      "                4,\n",
      "                7,\n",
      "                4,\n",
      "                6,\n",
      "                7,\n",
      "                1,\n",
      "                7,\n",
      "                7,\n",
      "                5,\n",
      "                4,\n",
      "                1,\n",
      "                3],\n",
      " 'Fluency': [9,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             5,\n",
      "             5,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             5,\n",
      "             8,\n",
      "             6,\n",
      "             7,\n",
      "             5,\n",
      "             5,\n",
      "             6,\n",
      "             5,\n",
      "             7,\n",
      "             5,\n",
      "             6,\n",
      "             7,\n",
      "             5,\n",
      "             6,\n",
      "             6,\n",
      "             6,\n",
      "             5,\n",
      "             6,\n",
      "             5,\n",
      "             5,\n",
      "             5,\n",
      "             6,\n",
      "             6,\n",
      "             9,\n",
      "             8,\n",
      "             7,\n",
      "             6,\n",
      "             9,\n",
      "             7,\n",
      "             5,\n",
      "             5,\n",
      "             7,\n",
      "             8,\n",
      "             8,\n",
      "             6,\n",
      "             7,\n",
      "             8,\n",
      "             9,\n",
      "             7,\n",
      "             5,\n",
      "             5,\n",
      "             7,\n",
      "             7,\n",
      "             5,\n",
      "             6,\n",
      "             5,\n",
      "             7,\n",
      "             5,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             9,\n",
      "             7,\n",
      "             6,\n",
      "             4,\n",
      "             7,\n",
      "             7,\n",
      "             5,\n",
      "             6,\n",
      "             5,\n",
      "             6,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             9,\n",
      "             5,\n",
      "             6,\n",
      "             6,\n",
      "             5,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             5,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             6,\n",
      "             5,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             5,\n",
      "             5,\n",
      "             8,\n",
      "             6,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             8,\n",
      "             5,\n",
      "             5,\n",
      "             5,\n",
      "             8,\n",
      "             4,\n",
      "             8,\n",
      "             5,\n",
      "             6,\n",
      "             7,\n",
      "             8,\n",
      "             5],\n",
      " 'Hard Persona Adherence': [6,\n",
      "                            6,\n",
      "                            8,\n",
      "                            6,\n",
      "                            5,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            7,\n",
      "                            3,\n",
      "                            7,\n",
      "                            9,\n",
      "                            6,\n",
      "                            7,\n",
      "                            2,\n",
      "                            4,\n",
      "                            7,\n",
      "                            9,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            9,\n",
      "                            7,\n",
      "                            7,\n",
      "                            9,\n",
      "                            8,\n",
      "                            8,\n",
      "                            9,\n",
      "                            7,\n",
      "                            6,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            7,\n",
      "                            6,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            8,\n",
      "                            6,\n",
      "                            6,\n",
      "                            7,\n",
      "                            6,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            9,\n",
      "                            8,\n",
      "                            9,\n",
      "                            5,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            4,\n",
      "                            6,\n",
      "                            6,\n",
      "                            6,\n",
      "                            7,\n",
      "                            6,\n",
      "                            9,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            9,\n",
      "                            7,\n",
      "                            6,\n",
      "                            6,\n",
      "                            7,\n",
      "                            6,\n",
      "                            6,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            4,\n",
      "                            6,\n",
      "                            5,\n",
      "                            5,\n",
      "                            7,\n",
      "                            7,\n",
      "                            4,\n",
      "                            5,\n",
      "                            8,\n",
      "                            8,\n",
      "                            5,\n",
      "                            6,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            6,\n",
      "                            4,\n",
      "                            5,\n",
      "                            7,\n",
      "                            5,\n",
      "                            5,\n",
      "                            4,\n",
      "                            9,\n",
      "                            7,\n",
      "                            4,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8],\n",
      " 'Self-consistency': [6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      3,\n",
      "                      6,\n",
      "                      9,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      9,\n",
      "                      7,\n",
      "                      8,\n",
      "                      6,\n",
      "                      4,\n",
      "                      3,\n",
      "                      7,\n",
      "                      9,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      9,\n",
      "                      8,\n",
      "                      6,\n",
      "                      8,\n",
      "                      9,\n",
      "                      9,\n",
      "                      9,\n",
      "                      8,\n",
      "                      3,\n",
      "                      8,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      9,\n",
      "                      8,\n",
      "                      8,\n",
      "                      7,\n",
      "                      3,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      8,\n",
      "                      8,\n",
      "                      6,\n",
      "                      7,\n",
      "                      9,\n",
      "                      8,\n",
      "                      8,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      8,\n",
      "                      7,\n",
      "                      6,\n",
      "                      9,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      5,\n",
      "                      7,\n",
      "                      8,\n",
      "                      6,\n",
      "                      6,\n",
      "                      8,\n",
      "                      6,\n",
      "                      9,\n",
      "                      6,\n",
      "                      8,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      8,\n",
      "                      6,\n",
      "                      7,\n",
      "                      9,\n",
      "                      9,\n",
      "                      6,\n",
      "                      9,\n",
      "                      6,\n",
      "                      6,\n",
      "                      5,\n",
      "                      6,\n",
      "                      6,\n",
      "                      9,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      9,\n",
      "                      7,\n",
      "                      8,\n",
      "                      7,\n",
      "                      6,\n",
      "                      5,\n",
      "                      1,\n",
      "                      5,\n",
      "                      6,\n",
      "                      9,\n",
      "                      1,\n",
      "                      9,\n",
      "                      7,\n",
      "                      8,\n",
      "                      2,\n",
      "                      7,\n",
      "                      8,\n",
      "                      6,\n",
      "                      7,\n",
      "                      8,\n",
      "                      6,\n",
      "                      8,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      9,\n",
      "                      7],\n",
      " 'Task Completion': [7,\n",
      "                     7,\n",
      "                     9,\n",
      "                     6,\n",
      "                     8,\n",
      "                     8,\n",
      "                     7,\n",
      "                     8,\n",
      "                     6,\n",
      "                     9,\n",
      "                     9,\n",
      "                     8,\n",
      "                     9,\n",
      "                     7,\n",
      "                     7,\n",
      "                     7,\n",
      "                     8,\n",
      "                     7,\n",
      "                     8,\n",
      "                     7,\n",
      "                     7,\n",
      "                     9,\n",
      "                     8,\n",
      "                     6],\n",
      " 'opinions_qty': [3,\n",
      "                  3,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  1,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  2,\n",
      "                  1,\n",
      "                  5,\n",
      "                  2,\n",
      "                  3,\n",
      "                  2,\n",
      "                  2,\n",
      "                  3,\n",
      "                  2]}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Proposition</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>Standard Deviation</th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hard Persona Adherence</td>\n",
       "      <td>6.625000</td>\n",
       "      <td>1.341406</td>\n",
       "      <td>120.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Self-consistency</td>\n",
       "      <td>6.791667</td>\n",
       "      <td>1.634172</td>\n",
       "      <td>120.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Fluency</td>\n",
       "      <td>6.375000</td>\n",
       "      <td>1.195668</td>\n",
       "      <td>120.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>opinions_qty</td>\n",
       "      <td>2.208333</td>\n",
       "      <td>0.779028</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Task Completion</td>\n",
       "      <td>7.583333</td>\n",
       "      <td>0.974308</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Divergence</td>\n",
       "      <td>4.916667</td>\n",
       "      <td>2.244155</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Proposition  Average Score  Standard Deviation  Count\n",
       "0  Hard Persona Adherence       6.625000            1.341406  120.0\n",
       "1        Self-consistency       6.791667            1.634172  120.0\n",
       "2                 Fluency       6.375000            1.195668  120.0\n",
       "3            opinions_qty       2.208333            0.779028   24.0\n",
       "4         Task Completion       7.583333            0.974308   24.0\n",
       "5              Divergence       4.916667            2.244155   24.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if experiment_runner.has_finished_all_experiments():\n",
    "    print(\"All experiments have been finished.\")\n",
    "    print(f\"STATISTICTS: Control vs\")\n",
    "    pprint(experiment_runner.run_statistical_tests(control_experiment_name='Control'))\n",
    "\n",
    "    # plot scores of both experiments\n",
    "    experiment_control_scores = experiment_runner.get_experiment_results(\"Control\")\n",
    "    experiment_treatment_scores = experiment_runner.get_experiment_results(\"Treatment\")\n",
    "    \n",
    "    \n",
    "    plot_scores(experiment_control_scores)\n",
    "    plot_scores(experiment_treatment_scores)\n",
    "\n",
    "else:\n",
    "    print(\"Not all experiments have been finished. RESTART AND RERUN.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experiment_runner.finish_active_experiment()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
